#include "Host.hpp"
#include "UrlFrontier.hpp"
#include "UrlManager.hpp"
#include "Crawl.hpp"
#include <iostream>
#include <iostream>

using namespace std;

int
main(int argc, char** argv)
{
    Host host;
    UrlManager url_manager("./tmp/UrlSeen.db");
    UrlFrontier url_frontier("./tmp/UrlToBeVisited", 10);
    Crawl crawl("./result/", 10);

    string tmp;
    vector<string> url_vec;
    while (cin >> tmp) {
        url_vec.push_back(tmp);
    }
    
    unsigned counter = 0;
    while (!url_vec.empty() && counter < 100) {
        for (unsigned i=0; i < url_vec.size(); ++i) {
            if (url_manager.UrlProcess(url_vec[i], host)) {
                url_frontier.AddUrl(url_vec[i]);
            } else {
                cerr << "Fail in url manager : " << url_vec[i] << endl;
            }
        }
        cout << "All url pushed in frontier!" << endl;
        vector<queue<Url> > task_queue;
        counter += url_frontier.GetTaskQueue(task_queue, host);
        cout << "counter: " << counter << endl;
        cout << "number of task queue : " << crawl.ImportTaskQueue(task_queue) << endl;
        crawl.StartCrawling();
        url_vec = crawl.GetUrlList();

        for (unsigned i=0; i<url_vec.size(); ++i) {
            cout << "To be fetched: " << url_vec[i] << endl;
        }
    }
}
